﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading;
using System.Xml;
using Common.Utilities;
using Common.Utilities.Dispatcher;

namespace Workflows.Components.TextMining.Clustering
{
    /// <summary>
    /// 
    /// </summary>
    public class FeatureSetWorker:IWorker
    {
        #region IWorker Members
        #region fields
        private Thread _WorkerThread;
        private FeatureSetInstruction _Instruction;
        private FeatureSetOutput _Output;
        private Guid _JobTicketID;
        private RunContext _Context;
        private string _WorkflowName;
        private string _ComponentName;

        /// <summary>
        /// 
        /// </summary>
        public string ComponentName
        {
            get { return _ComponentName; }
        }

        /// <summary>
        /// 
        /// </summary>
        public string WorkflowName
        {
            get { return _WorkflowName; }
        }
        #endregion

        public Thread WorkerThread
        {
            get { return _WorkerThread; }
        }

        public InstructionBase Instruction
        {
            get { return _Instruction; }
        }

        public ResultBase Output
        {
            get { return _Output; }
        }

        public Type ComponentType
        {
            get { return typeof(FeatureSetComponent); }
        }

        public RunContext Context
        {
            get { return _Context; }
            set { _Context = value; }
        }

        public void SetInstruction(Guid ticketGuid, InstructionBase request, string workflowName, string componentName)
        {
            this._JobTicketID = ticketGuid;
            this._Instruction = (FeatureSetInstruction)request;
            this._WorkflowName = workflowName;
            this._ComponentName = componentName;
        }

        public void SetInstruction(RunContext context, Guid ticketGuid, InstructionBase request, string workflowName, string componentName)
        {
            this._JobTicketID = ticketGuid;
            this._Instruction = (FeatureSetInstruction)request;
            this._Context = context;
            this._WorkflowName = workflowName;
            this._ComponentName = componentName;
        }

        public void DoWork()
        {
            if (this.WorkerStartedEvent != null)
            {
                this.WorkerStartedEvent(this._JobTicketID, this, this.GetType(), this._Instruction);
            }
            this._WorkerThread = Thread.CurrentThread;
            this.StartTraining();
        }

        public void DoWorkAsync()
        {
            if (this.WorkerStartedEvent != null)
            {
                this.WorkerStartedEvent(this._JobTicketID, this, this.GetType(), this._Instruction);
            }
            this._WorkerThread = new Thread(new ThreadStart(this.StartTraining));
            this._WorkerThread.Start();
        }

        public void CancelWork()
        {
            if (this._WorkerThread != null && this._WorkerThread.IsAlive)
            {
                this._WorkerThread.Abort();
                if (this.WorkerCancelledEvent != null)
                {
                    this.WorkerCancelledEvent(this._JobTicketID, this.GetType(), this._Instruction);
                }
            }

        }

        public event WorkerStarted WorkerStartedEvent;
        public event WorkerFinished WorkerFinishedEvent;
        public event WorkerHasError WorkerErredEvent;
        public event WorkerCancelled WorkerCancelledEvent;
        public event WorkerStatusChanged WorkerStatusChangedEvent;

        #endregion

        #region variables that do not change between iterations
        private Dictionary<int,string> _TermPhrases;
        /// <summary></summary>
        public Dictionary<int,string> TermPhrases
        {
            get
            {
                return this._TermPhrases;
            }
            set
            {
                this._TermPhrases = value;
            }
        }

        private Dictionary<int,Dictionary<int,int>> _DocTermFrequencies;
        /// <summary>docid -> (termid -> freq) </summary>
        public Dictionary<int,Dictionary<int,int>> DocTermFrequencies
        {
            get
            {
                return this._DocTermFrequencies;
            }
            set
            {
                this._DocTermFrequencies = value;
            }
        }

        private Dictionary<int,Dictionary<int,double>> _TermDocTfidfs;
        /// <summary></summary>
        public Dictionary<int,Dictionary<int,double>> TermDocTfidfs
        {
            get
            {
                return this._TermDocTfidfs;
            }
            set
            {
                this._TermDocTfidfs = value;
            }
        }

        private double _ExpectedOverallPerformanceScore;
        /// <summary></summary>
        public double ExpectedOverallPerformanceScore
        {
            get
            {
                return this._ExpectedOverallPerformanceScore;
            }
            set
            {
                this._ExpectedOverallPerformanceScore = value;
            }
        }
        #endregion

        #region iteration variables stored in memory
        private int _CurrentIteration;
        /// <summary></summary>
        public int CurrentIteration
        {
            get
            {
                return this._CurrentIteration;
            }
            set
            {
                this._CurrentIteration = value;
            }
        }

        private double _LastOverallPerformanceScore;
        /// <summary></summary>
        public double LastOverallPerformanceScore
        {
            get
            {
                return this._LastOverallPerformanceScore;
            }
            set
            {
                this._LastOverallPerformanceScore = value;
            }
        }

        private double _CurrentOverallPerformanceSocre;
        /// <summary></summary>
        public double CurrentOverallPerformanceSocre
        {
            get
            {
                return this._CurrentOverallPerformanceSocre;
            }
            set
            {
                this._CurrentOverallPerformanceSocre = value;
            }
        }

        private Dictionary<int,double> _ClusterDistances;
        /// <summary></summary>
        public Dictionary<int,double> ClusterDistances
        {
            get
            {
                return this._ClusterDistances;
            }
            set
            {
                this._ClusterDistances = value;
            }
        }

        private Dictionary<int,int> _DocClusters;
        /// <summary></summary>
        public Dictionary<int,int> DocClusters
        {
            get
            {
                return this._DocClusters;
            }
            set
            {
                this._DocClusters = value;
            }
        }

        private Dictionary<int,List<int>> _ClusterDocs;
        /// <summary>clusterid -> list of docids</summary>
        public Dictionary<int,List<int>> ClusterDocs
        {
            get
            {
                return this._ClusterDocs;
            }
            set
            {
                this._ClusterDocs = value;
            }
        }

        private Dictionary<int,Dictionary<int,double>> _ClusterClusterDistances;
        /// <summary></summary>
        public Dictionary<int,Dictionary<int,double>> ClusterClusterDistances
        {
            get
            {
                return this._ClusterClusterDistances;
            }
            set
            {
                this._ClusterClusterDistances = value;
            }
        }

        private Dictionary<int,TermPerformance> _TermClassifierScores;
        /// <summary></summary>
        public Dictionary<int, TermPerformance> TermClassifierScores
        {
            get
            {
                return this._TermClassifierScores;
            }
            set
            {
                this._TermClassifierScores = value;
            }
        }

        private Dictionary<int,double> _TermWeights;
        /// <summary></summary>
        public Dictionary<int,double> TermWeights
        {
            get
            {
                return this._TermWeights;
            }
            set
            {
                this._TermWeights = value;
            }
        }

        private int _MaxIteration = 10;
        /// <summary></summary>
        public int MaxIteration
        {
            get
            {
                return this._MaxIteration;
            }
            set
            {
                this._MaxIteration = value;
            }
        }
        #endregion

        #region iteration
        private void StartTraining()
        {
            try
            {
                this._CurrentIteration = 0;
                this.ReadDocTerms(this._Instruction.StartingClusterResultFilePath);
                this._TermWeights = this.ReadTermWeights();
                int termsUpdated = 1;
                while (this._CurrentIteration < this._MaxIteration && 
                    termsUpdated>0)
                {
                    this._CurrentIteration += 1;
                    this.TryClusterDocuments();
                    this.UpdateCurrentPerformances();
                    termsUpdated = this.UpdateTermWeights(ref this._TermWeights);
                }
                if(this.WorkerFinishedEvent !=null)
                {
                    this.WorkerFinishedEvent(this._JobTicketID, this.GetType(), this._Instruction, null);
                }
            }
            catch (Exception ex)
            {
                if(this.WorkerErredEvent !=null)
                {
                    this.WorkerErredEvent(this._JobTicketID, this.GetType(), this._Instruction, ex.ToString(),true);
                }
            }
        }
        #endregion

        #region static data initialization
        private void ReadDocTerms(string xmlFilePath)
        {
            this._DocTermFrequencies = ClusterResultReader.ReadDocTermFrequencies(xmlFilePath);
            this._TermDocTfidfs = ClusterResultReader.ReadTermTfidfForDocuments(xmlFilePath);
            Dictionary<int, TermPerformance> termPerformances = ClusterResultReader.ReadTermPerformance(xmlFilePath);
            this._TermPhrases=new Dictionary<int, string>();
            double startingPerformance = 0;
            //this._ExpectedOverallPerformanceScore = 0;
            double maxTermPerformance = double.NegativeInfinity;
            double minTermPerformance = double.PositiveInfinity;
            foreach(int termID in termPerformances.Keys)
            {
                TermPerformance termPerformance = termPerformances[termID];
                this._TermPhrases.Add(termID,termPerformance.Phrase);
                if(termPerformance.ClassifierScore > maxTermPerformance)
                    maxTermPerformance = termPerformance.ClassifierScore;
                if(termPerformance.ClassifierScore < minTermPerformance)
                    minTermPerformance = termPerformance.ClassifierScore;
            }
            double termPerformanceRange = maxTermPerformance - minTermPerformance;
            foreach (int termID in termPerformances.Keys)
            {
                TermPerformance termPerformance = termPerformances[termID];
                startingPerformance += termPerformance.ClassifierScore;
                if (termPerformance.ClassifierScore > termPerformanceRange / 2)
                    this._ExpectedOverallPerformanceScore += Math.Max(1, maxTermPerformance);
            }

            if(File.Exists(this._Instruction.IterationFilePath))
            {
                File.Delete(this._Instruction.IterationFilePath);
            }
            XmlDocument xDoc = new XmlDocument();
            xDoc.LoadXml("<Iterations></Iterations>");
            XmlNode root = xDoc.DocumentElement;
            XmlDataUtil.UpdateAttribute(ref xDoc, root, "StartingPerformanceScore", startingPerformance.ToString());
            XmlDataUtil.UpdateAttribute(ref xDoc, root, "EndingPerformanceScore", this._ExpectedOverallPerformanceScore.ToString());
            XmlDataUtil.UpdateAttribute(ref xDoc, root, "StartingClusterResultFilePath",xmlFilePath);
            xDoc.Save(this._Instruction.IterationFilePath);
        }
        #endregion

        #region term performance
        private Dictionary<int,double> ReadTermWeights()
        {
            Dictionary<int,double> termWeights=new Dictionary<int, double>();
            if(this.CurrentIteration>0)
            {
                string currentClusterFilePath = string.Format(this._Instruction.ClusterResultFileNamePattern,
                                                              this.CurrentIteration);
                if(!File.Exists(currentClusterFilePath))
                    throw new Exception("Unable to find cluster result file");
                termWeights = ClusterResultReader.ReadTermWeights(currentClusterFilePath);
            }
            else
            {
                foreach(int termID in this.TermPhrases.Keys)
                {
                    termWeights.Add(termID,1.0);
                }
            }

            XmlDocument xDoc = new XmlDocument();
            if (File.Exists(this._Instruction.DocumentTermListFilePath))
            {
                xDoc.Load(this._Instruction.DocumentTermListFilePath);
            }
            else
            {
                xDoc.LoadXml("<Docs></Docs>");
            }
            XmlNode root = xDoc.DocumentElement;
            foreach(int docID in this.DocTermFrequencies.Keys)
            {
                XmlNode docNode = xDoc.SelectSingleNode("Docs/Doc[@DocID=\"" + docID + "\"]");
                if(docNode==null)
                {
                    docNode = XmlDataUtil.AddElement(ref xDoc, root, "Doc");
                    XmlDataUtil.UpdateAttribute(ref xDoc, docNode, "DocID", docID.ToString());
                    XmlDataUtil.UpdateAttribute(ref xDoc, docNode, "Label", "");
                }
                Dictionary<int, int> termFreqsForDoc = this.DocTermFrequencies[docID];
                foreach(int termID in termFreqsForDoc.Keys)
                {
                    double weight = termWeights[termID];
                    XmlNode termNode = docNode.SelectSingleNode("Term[@TermID=\"" + termID + "\"]");
                    if (termNode == null)
                    {
                        termNode = XmlDataUtil.AddElement(ref xDoc, docNode, "Term");
                        XmlDataUtil.UpdateAttribute(ref xDoc, termNode, "TermID", termID.ToString());
                        string phrase = this.TermPhrases[termID];
                        XmlDataUtil.UpdateAttribute(ref xDoc, termNode, "Phrase", phrase);
                        int count = termFreqsForDoc[termID];
                        XmlDataUtil.UpdateAttribute(ref xDoc, termNode, "Count", count.ToString());
                        XmlDataUtil.UpdateAttribute(ref xDoc, termNode, "Weight", weight.ToString());
                    }
                    else
                    {
                        termNode.Attributes.GetNamedItem("Weight").Value = weight.ToString();
                    }
                }
            }
            xDoc.Save(this._Instruction.DocumentTermListFilePath);

            return termWeights;
        }

        private void UpdateCurrentPerformances()
        {
            string xmlFilePath = string.Format(this._Instruction.ClusterResultFileNamePattern, this._CurrentIteration);
            if(!File.Exists(xmlFilePath))
                throw new Exception("Unable to find cluster file");

            this._ClusterDistances = ClusterResultReader.ReadClusterDistances(xmlFilePath);
            this._DocClusters = ClusterResultReader.ReadDocClusters(xmlFilePath);
            this._ClusterDocs = ClusterResultReader.ReadClusterDocs(xmlFilePath);
            this._ClusterClusterDistances = ClusterResultReader.ReadClusterClusterDistances(xmlFilePath);
            this._TermClassifierScores = ClusterResultReader.ReadTermPerformance(xmlFilePath);
            this._LastOverallPerformanceScore = this._CurrentOverallPerformanceSocre;
            this._CurrentOverallPerformanceSocre = 0;
            foreach(int clusterID in this._ClusterDistances.Keys)
            {
                if(this._ClusterDocs.ContainsKey(clusterID))
                {
                    int docCountForCluster = this._ClusterDocs[clusterID].Count;
                    double clusterDistance = Math.Max(0.01, this._ClusterDistances[clusterID]);
                    this._CurrentOverallPerformanceSocre += (double) docCountForCluster/clusterDistance;    
                }
            }
            //foreach(int termID in this._TermClassifierScores.Keys)
            //{
            //    this._CurrentOverallPerformanceSocre += this._TermClassifierScores[termID].ClassifierScore;
            //}

            XmlDocument xDoc=new XmlDocument();
            xDoc.Load(this._Instruction.IterationFilePath);
            XmlNode root = xDoc.DocumentElement;
            XmlNode iterationNode = XmlDataUtil.AddElement(ref xDoc, root, "Iteration");
            XmlDataUtil.UpdateAttribute(ref xDoc, iterationNode,"Number",this.CurrentIteration.ToString());
            XmlDataUtil.UpdateAttribute(ref xDoc, iterationNode, "OverallPerformanceScore",this.CurrentOverallPerformanceSocre.ToString());
            XmlDataUtil.UpdateAttribute(ref xDoc, iterationNode, "ClusterResultFilePath",xmlFilePath);
            xDoc.Save(this._Instruction.IterationFilePath);
        }
        #endregion

        #region back propagation
        /// <summary>
        /// also save term weight changes to DocumentTermList file
        /// </summary>
        /// <param name="termWeights"></param>
        /// <returns>number of terms updated</returns>
        private int UpdateTermWeights(ref Dictionary<int,double> termWeights)
        {
            List<int> frozenTermIDs=new List<int>();
            XmlDocument xDoc = new XmlDocument();
            if (File.Exists(this._Instruction.TermWeightFilePath))
            {
                xDoc.Load(this._Instruction.TermWeightFilePath);
            }
            else
            {
                xDoc.LoadXml("<Terms></Terms>");
            }
            XmlNodeList frozenTermNodes = xDoc.SelectNodes("//Term[@IsFrozen=\"" + true.ToString() + "\"]");
            if(frozenTermNodes !=null && frozenTermNodes.Count>0)
            {
                foreach(XmlNode frozenTermNode in frozenTermNodes)
                {
                    int termID = int.Parse(frozenTermNode.Attributes.GetNamedItem("TermID").Value);
                    if(!frozenTermIDs.Contains(termID))
                        frozenTermIDs.Add(termID);
                }
            }

            int[] termIDs=new int[termWeights.Count];
            termWeights.Keys.CopyTo(termIDs,0);
            double minScore = double.PositiveInfinity;
            double maxScore = double.NegativeInfinity;
            foreach(TermPerformance termPerformance in this.TermClassifierScores.Values)
            {
                if(termPerformance.ClassifierScore>maxScore)
                    maxScore = termPerformance.ClassifierScore;
                if(termPerformance.ClassifierScore<minScore)
                    minScore = termPerformance.ClassifierScore;
            }
            double filterScore1 = minScore + (maxScore - minScore) * 0.1;
            double filterScore2 = minScore + (maxScore - minScore) * 0.9;
            foreach (int termID in this.TermClassifierScores.Keys)
            {
                if (this.TermClassifierScores[termID].ClassifierScore < filterScore1)
                {
                    termWeights[termID] = 0.0;
                    if (!frozenTermIDs.Contains(termID))
                        frozenTermIDs.Add(termID);
                }
                else if(this.TermClassifierScores[termID].ClassifierScore > filterScore2)
                {
                    termWeights[termID] = 1.0;
                    if (!frozenTermIDs.Contains(termID))
                        frozenTermIDs.Add(termID);
                }
            }
            List<TermPerformance> candidateTermPerformances =new List<TermPerformance>();
            foreach (int termID in this.TermClassifierScores.Keys)
            {
                if (!frozenTermIDs.Contains(termID))
                {
                    candidateTermPerformances.Add(this.TermClassifierScores[termID]);
                }
            }
            candidateTermPerformances.Sort();
            int quantileCount = (int)candidateTermPerformances.Count / 4;
            for (int i = 0; i < quantileCount; i++)
            {
                if (candidateTermPerformances.Count > 0)
                {
                    TermPerformance termPerformance = candidateTermPerformances[0];
                    int termID = termPerformance.TermID;
                    termWeights[termID] = 1.0;
                    candidateTermPerformances.RemoveAt(0);
                }
            }
            for (int i = 0; i < quantileCount; i++)
            {
                if (candidateTermPerformances.Count > 0)
                {
                    TermPerformance termPerformance = candidateTermPerformances[candidateTermPerformances.Count - 1];
                    int termID = termPerformance.TermID;
                    termWeights[termID] = 0.0;
                    candidateTermPerformances.RemoveAt(candidateTermPerformances.Count - 1);
                }
            }

            int termsUpdated = 0;
            // adjust term weights 
            if(this.WorkerStatusChangedEvent !=null)
            {
                string msg = string.Format("Iteration #{0}: updating weights of {1} terms...", this.CurrentIteration,
                                           candidateTermPerformances.Count);
                int pct = this.CurrentIteration%100;
                this.WorkerStatusChangedEvent(this._JobTicketID, this.GetType(), this._Instruction, this.WorkflowName, this.ComponentName, msg, pct);
            }

            //double deltaWeight = 10.00/candidateTermPerformances.Count;
            //double startingWeight = 100;
            foreach(TermPerformance termPerformance in candidateTermPerformances)
            {
                int termID = termPerformance.TermID;
                
                double deltaClusterDistance = this.GetTermClusterDistanceDelta(termID);
                if (deltaClusterDistance != 0)
                    termsUpdated += 1;
                double newTermWeight = this.GetNewTermWeight(termWeights[termID], deltaClusterDistance);
                termWeights[termID] = newTermWeight;
                
                if(this.WorkerStatusChangedEvent !=null && termsUpdated % 10==0)
                {
                    int pct = this._CurrentIteration*100/this._MaxIteration;
                    string msg = string.Format("Iteration #{0}: updating term #{1} of {2}...Current performance is:{3}",
                                               this.CurrentIteration, termsUpdated, candidateTermPerformances.Count,
                                               this._CurrentOverallPerformanceSocre);
                    this.WorkerStatusChangedEvent(this._JobTicketID, this.GetType(), this._Instruction, this.WorkflowName, this.ComponentName, msg, pct);
                }
            }

            // save result
            XmlNode root = xDoc.DocumentElement;
            foreach(int termID in termWeights.Keys)
            {
                double weight = termWeights[termID];
                double classifierScore = this.TermClassifierScores[termID].ClassifierScore;

                XmlNode termNode = xDoc.SelectSingleNode("//Term[@TermID=\"" + termID + "\"]");
                if(termNode==null)
                {
                    termNode = XmlDataUtil.AddElement(ref xDoc, root, "Term");
                    XmlDataUtil.UpdateAttribute(ref xDoc, termNode,"TermID",termID.ToString());
                    string phrase = this.TermPhrases[termID];
                    XmlDataUtil.UpdateAttribute(ref xDoc, termNode,"Phrase",phrase);
                    bool isFrozen = false;
                    if(frozenTermIDs.Contains(termID))
                        isFrozen = true;
                    XmlDataUtil.UpdateAttribute(ref xDoc, termNode,"IsFrozen",isFrozen.ToString());
                    
                    XmlDataUtil.UpdateAttribute(ref xDoc, termNode, "CurrentWeight",weight.ToString());
                }

                XmlNode iterationNode = XmlDataUtil.AddElement(ref xDoc, termNode, "Iteration");
                XmlDataUtil.UpdateAttribute(ref xDoc, iterationNode,"Number",this.CurrentIteration.ToString());
                XmlDataUtil.UpdateAttribute(ref xDoc,iterationNode,"Weight",weight.ToString());
                XmlDataUtil.UpdateAttribute(ref xDoc, iterationNode,"ClassifierScore",classifierScore.ToString());
            }
            xDoc.Save(this._Instruction.TermWeightFilePath);

            XmlDocument termListDoc = new XmlDocument();
            termListDoc.Load(this._Instruction.DocumentTermListFilePath);
            XmlNodeList termNodes = termListDoc.SelectNodes("//Term");
            foreach(XmlNode termNode in termNodes)
            {
                int termID = int.Parse(termNode.Attributes.GetNamedItem("TermID").Value);
                if(termWeights.ContainsKey(termID))
                {
                    double weight = termWeights[termID];
                    termNode.Attributes.GetNamedItem("Weight").Value = weight.ToString();
                }
            }
            termListDoc.Save(this._Instruction.DocumentTermListFilePath);

            return termsUpdated;
        }

        private double GetTermClusterDistanceDelta(int termID)
        {
            if (!this.TermDocTfidfs.ContainsKey(termID))
                return 0;
            Dictionary<int, double> termFreqsForDoc = this.TermDocTfidfs[termID];
            List<int> positiveClusterIDs=new List<int>();
            foreach(int docID in termFreqsForDoc.Keys)
            {
                int clusterID = this.DocClusters[docID];
                if(!positiveClusterIDs.Contains(clusterID))
                    positiveClusterIDs.Add(clusterID);
            }
            double avgClusterDistance = 0;
            foreach(int clusterID in positiveClusterIDs)
            {
                avgClusterDistance += this.ClusterDistances[clusterID];
            }
            double newClusterDistance = 0;
            int[] positiveDocIDs=new int[termFreqsForDoc.Count];
            termFreqsForDoc.Keys.CopyTo(positiveDocIDs,0);
            for(int i=0;i<positiveDocIDs.Length;i++)
            {
                int docID1 = positiveDocIDs[i];
                for(int k=i+1;k<positiveDocIDs.Length;k++)
                {
                    int docID2 = positiveDocIDs[k];
                    newClusterDistance += this.CalculateDocDocDistances(docID1, docID2, TermWeights);
                }
            }
            newClusterDistance = newClusterDistance/positiveDocIDs.Length;
            return avgClusterDistance - newClusterDistance;
        }

        private double GetNewTermWeight(double oldWeight, double deltaClusterDifference)
        {
            if(deltaClusterDifference==0)
                return oldWeight;

            double adjustment = 1/(1 + Math.Pow(Math.E, (-Math.Abs(deltaClusterDifference))));
            if (deltaClusterDifference > 0)
                return Math.Min(1.0, oldWeight*(1 + adjustment));
            else
                return Math.Max(0.0, oldWeight * (1 - adjustment));
        }
        #endregion

        #region run clustering
        private void TryClusterDocuments()
        {
            if(this.WorkerStatusChangedEvent !=null)
            {
                string msg = string.Format("Iteration {0}: clustering...", this.CurrentIteration);
                int percent = this.CurrentIteration%this.CurrentIteration;
                this.WorkerStatusChangedEvent(this._JobTicketID, this.GetType(), this._Instruction, this.WorkflowName, this.ComponentName, msg, percent);
            }

            string xmlFilePath = string.Format(this._Instruction.ClusterResultFileNamePattern, this._CurrentIteration);

            UnsupervisedClusteringComponent clusteringComponent =
                (UnsupervisedClusteringComponent) Activator.CreateInstance(typeof (UnsupervisedClusteringComponent));
            UnsupervisedClusteringInstruction clusteringInstruction =
                new UnsupervisedClusteringInstruction(
                    clusteringComponent.ComponentID, Guid.NewGuid(), 
                    this._Instruction.DocumentTermListFilePath,
                    0.35, 0.25, 0.05, new List<string>(), 
                    PurityMeasureMethod.EntropyMeasure,
                    DistanceCalculationMethod.WeightedSum,
                    DocumentClusteringMethod.HierarchicalClustering,
                    false, 1.50, xmlFilePath, TermFuzzyMatchMethod.None);

            UnsupervisedClusteringWorker clusteringWorker=new UnsupervisedClusteringWorker();
            clusteringWorker.SetInstruction(Guid.NewGuid(),clusteringInstruction, this.WorkflowName, this.ComponentName);
            clusteringWorker.DoWork();
        }

        private double CalculateDocDocDistances(int docID1, int docID2, Dictionary<int,double> termWeights)
        {
            Dictionary<int,int> contributingTerms=new Dictionary<int, int>();
            WeightedTermSimilarityCalculator calculator=new WeightedTermSimilarityCalculator();
            return calculator.CalculateDistance(
                this.GetDocTermFreqs(docID1, termWeights),
                this.GetDocTermFreqs(docID2, termWeights),
                termWeights, false, 0, ref contributingTerms);
        }

        private Dictionary<int, DocTermFreq> GetDocTermFreqs(int docID, Dictionary<int,double> termWeights)
        {
            Dictionary<int, DocTermFreq> docTermFreqs = new Dictionary<int, DocTermFreq>();
            Dictionary<int, int> termFreqs = this.DocTermFrequencies[docID];
            foreach(int termID in termFreqs.Keys)
            {
                DocTermFreq docTermFreq=new DocTermFreq();
                docTermFreq.TermID = termID;
                docTermFreq.Count = termFreqs[termID];
                docTermFreq.Phrase = this.TermPhrases[termID];
                docTermFreq.Weight = termWeights[termID];
                docTermFreqs.Add(termID, docTermFreq);
            }
            return docTermFreqs;
        }
        #endregion
    }
}
